package icu.smile.proxy.task;

import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.logging.Logger;

/**
 * <p>
 * 爬虫处理器
 * </p>
 *
 * @author starrysky
 * @since 2021/6/6
 */
@Component
public class MiPuPageProcessor implements PageProcessor {

    private static final Logger LOGGER = Logger.getLogger(MiPuPageProcessor.class.getName());

    private static String cmdPrefix = "tesseract ";
    private static String cmdSuffix = " stdout";
    private static Process process = null;
    private static BufferedReader bufferedReader = null;
    private static String ImageResultOCR = null;

    private Site site = Site.me()
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36")
            .setDomain("proxy.mimvp.com")
            .addCookie("UMIMVPSESSID", "hct11388aottkqvlgkoecsqna6")
            .addCookie("Hm_lvt_51e3cc975b346e7705d8c255164036b3", "1622948301")
            .addCookie("Hm_lpvt_51e3cc975b346e7705d8c255164036b3", "1622949348")
            .setCharset("UTF-8")
            .setTimeOut(5000)
            .setRetrySleepTime(1000)
            .setRetryTimes(3);

    /***
     * <p>
     *     总的处理方法
     * </p>
     * @author starrysky
     * @since 2021/6/6 22:14
     * @param    page    page页面
     * @return void  无返回值
     */
    @Override
    public void process(Page page) {

        final List<String> proxyTabes = proxyTabes(page);
        page.addTargetRequests(proxyTabes);
        page.putField("ip", page.getHtml().xpath("//td[@class='free-proxylist-tbl-proxy-ip']/text()").all());
        page.putField("port", proxyPort(page));
        page.putField("type", page.getHtml().xpath("//td[@class='free-proxylist-tbl-proxy-type']/text()").all());
        page.putField("anonymous", page.getHtml().xpath("//td[@class='free-proxylist-tbl-proxy-anonymous']/text()").all());
        page.putField("location", fixLocation(page));
        page.putField("operator", fixOperator(page));
        page.putField("responseTime", page.getHtml().xpath("//td[@class='free-proxylist-tbl-proxy-pingtime']/@title").all());
        page.putField("transmissionTime", page.getHtml().xpath("//td[@class='free-proxylist-tbl-proxy-transfertime']/@title").all());
        page.putField("verificationTime", page.getHtml().xpath("//td[@class='free-proxylist-tbl-proxy-checkdtime']/text()").all());
        page.addTargetRequests(fixSmileUrl(page));
        page.addTargetRequests(fixListnav(page));
    }

    /***
     * <p>
     *     将page中的三个代理分类的url抽出
     * </p>
     * @author starrysky
     * @since 2021/6/6 22:11
     * @param    page    page页面
     * @return java.util.List<java.lang.String> 存储三个代理方式的分类url
     */
    public List<String> proxyTabes(Page page) {
        List<String> tabes = new ArrayList<>();
        for (String value : page.getHtml().css("div.free-proxytype-tabs").xpath("//a/@href").all()) {
            tabes.add(page.getUrl().toString().substring(0, 23) + value);
        }
        return tabes;
    }

    /***
     * <p>
     *     将页面中的记录着代理端口的图片抽出
     * </p>
     * @author starrysky
     * @since 2021/6/6 22:12
     * @param    page    page页面
     * @return java.util.List<java.lang.Integer> 存储转换之后的端口
     */
    public List<Integer> proxyPort(Page page) {
        List<Integer> protoPort = new ArrayList<>();
        for (String value : page.getHtml().xpath("//td[@class='free-proxylist-tbl-proxy-port']//img/@src").all()) {
            protoPort.add(convertImageInteger(page.getUrl().toString().substring(0, 23) + value));
        }
        return protoPort;
    }

    /***
     * <p>
     *     将从图片中提取出来
     * </p>
     * @author starrysky
     * @since 2021/6/6 22:13
     * @param    url    每一个记录着端口的图片url
     * @return java.lang.Integer 端口号
     */
    public synchronized Integer convertImageInteger(String url) {
        try {
            process = Runtime.getRuntime().exec(cmdPrefix + url + cmdSuffix);
            bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()));
            ImageResultOCR = bufferedReader.readLine()
                    .replace(",", "")
                    .replace(".", "")
                    .replace("!", "")
                    .replace("@", "")
                    .replace("#", "")
                    .replace("$", "")
                    .replace("%", "")
                    .replace("^", "")
                    .replace("&", "")
                    .replace("*", "");
            //处理8080被识别成为B080情况
            if (ImageResultOCR.contains("B") && ImageResultOCR.length() != 3) {
                ImageResultOCR = ImageResultOCR.replace("B", "8");
                //处理三位的端口其本来为两位，去除被干扰的B，比如B80，这个方法有点风险后期会改进
            } else if (ImageResultOCR.contains("B") && ImageResultOCR.length() == 3) {
                ImageResultOCR = ImageResultOCR.replace("B", "");
            } else if (ImageResultOCR.contains("s") && ImageResultOCR.contains("e")) {
                ImageResultOCR = ImageResultOCR.replace("s", "5").replace("e", "2");
            } else if (ImageResultOCR.contains("s")) {
                ImageResultOCR = ImageResultOCR.replace("s", "5");
            } else if (ImageResultOCR.contains("e")) {
                ImageResultOCR = ImageResultOCR.replace("e", "2");
            }
            return (Integer) Integer.parseInt(ImageResultOCR);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (Exception e) {
            ImageResultOCR = "0";
            LOGGER.info("OCR识别出错,将使用0填充端口项目.");
        }
        return null;
    }

    public List<String> fixLocation(Page page) {
        List<String> location = new ArrayList<>();
        for (String loc : page.getHtml().xpath("//td[@class='free-proxylist-tbl-proxy-country']/text()").all()) {
            location.add(loc.replace("(", "").replace(")", "").trim());
        }
        return location;
    }

    /***
     * <p>
     *     修复运营商描述
     * </p>
     * @author starrysky
     * @since 2021/6/6 23:22
     * @param    page    page页面
     * @return java.util.List<java.lang.String> 返回运营商的描述
     */
    public List<String> fixOperator(Page page) {
        List<String> operator = new ArrayList<>();
        for (String oper : page.getHtml().xpath("//td[@class='free-proxylist-tbl-proxy-isp']/text()").all()) {
            operator.add(oper == null ? "暂无运营商" : oper);
        }
        return operator;
    }

    /***
     * <p>
     *     处理列表页面
     * </p>
     * @author starrysky
     * @since 2021/6/6 23:23
     * @param        page page页面
     * @return java.util.List<java.lang.String> 列表页面
     */
    public List<String> fixListnav(Page page) {
        List<String> listnv = new ArrayList<>();
        //判断列表寻址中包含...则需要根据收src和尾部src数字生成List
        if (page.getHtml().css("div#listnav").css("ul").xpath("//li/text()").all().contains("...")) {
            //构造条件【/freesecret?proxy，in_hp&sort=&pag，1】
            final String[] origin = page.getHtml().css("div#listnav").css("ul").xpath("//li//a/@href").all().get(0).toString().split("=");
//            【/freesecret?proxy，in_hp&sort=&pag]
            final String[] urlBody = Arrays.copyOf(origin,origin.length-1);
            final String[] firstElement = page.getHtml().css("div#listnav").css("ul").xpath("//li//a/@href").all().get(0).toString().split("=");
            //获取1 这个id
            int firstId = Integer.parseInt(firstElement[firstElement.length-1]);
            final String[] lastElement = page.getHtml().css("div#listnav").css("ul").xpath("//li//a/@href").all().get(page.getHtml().css("div#listnav").css("ul").xpath("//li//a/@href").all().size() - 1).toString().split("=");
            //获取最后一个48 这个id
            int lastId = Integer.parseInt(lastElement[lastElement.length-1]);
//            使用firstId、lastId构造for生成全部的url
            for (int i = firstId; i <= lastId ; i++) {
                listnv.add("https://proxy.mimvp.com/"+StringUtils.join(urlBody,"=")+"="+i);
            }
            return listnv;
        }
        //没有...说明页面少，直接提取生成完整url
        for (String url:page.getHtml().css("div#listnav").css("ul").xpath("//li//a/@href").all()){
            listnv.add("https://proxy.mimvp.com/"+url);
        }
        return listnv;
    }

    public List<String> fixSmileUrl(Page page){
        List<String> smileurl = new ArrayList<>();
        for (String url:page.getHtml().xpath("//div[@class='free-httptype-tabs']//a/@href").all()){
            smileurl.add("https://proxy.mimvp.com/"+url);
        }
        return smileurl;
    }
    @Override
    public Site getSite() {
        return site;
    }
}
